package com.terren.spider.core.html.scheme.impl;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.lang3.StringUtils;

import com.terren.spider.entity.biz.ArticleSearch;
import com.terren.spider.entity.core.Entry;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

public class SogouNewsScheme extends BasicCustomScheme{

	@Override
	public List<ArticleSearch> findTitleInfo(Page page, Entry entry) {
		List<ArticleSearch> articleList = new ArrayList<>();
		//----------------------------------爬取搜索页面的标题，发布时间，url
		Html pageHtml = page.getHtml();
		List<String> partList = pageHtml.xpath("//div[@class='vrwrap']").all();//待爬取的新闻列表(比实际多一条新闻推荐是无效的)
		Integer newsNum = partList.size()-1;//剔除掉多余的链接
		//Integer pageNum = Integer.parseInt(pageHtml.xpath("//*[@id='page']/strong/span[2]/text()").get());
		for(int i=0;i<newsNum;i++){
			ArticleSearch temp = new ArticleSearch();
			//Integer id =(pageNum-1)*partList.size()+i;
			String title = pageHtml.xpath("//*[@id='uigs_"+i+"']/text()").get()!=null?
					pageHtml.xpath("//*[@id='uigs_"+i+"']/text()").get():pageHtml.xpath("//*[@id='main']/div/div["+(i+1)+"]/div/h3/allText()").get();
			/*if(title==null){
				title= pageHtml.xpath("//*[@id='main']/div/div["+(i+1)+"]/div/h3/a/text()").get();
			}*/
			
			String authorClass=pageHtml.xpath("//*[@id='main']/div/div["+(i+1)+"]/div/div/div/p[1]/text()").get().trim();
			String url = pageHtml.xpath("//*[@id='uigs_"+i+"']").links().get()!=null?
					pageHtml.xpath("//*[@id='uigs_"+i+"']").links().get():pageHtml.xpath("//*[@id='main']/div/div["+(i+1)+"]/div/h3").links().get();
			temp.setTitle(title);
			temp.setNewsSource(StringUtils.substringBefore(authorClass, " "));
			temp.setPubTimeStr(StringUtils.substringAfter(authorClass, " "));
			if("".equals(temp.getPubTimeStr())||temp.getPubTimeStr()==null){
				temp.setNewsSource("");
				temp.setPubTimeStr(authorClass);
			}
			temp.setUrl(url);
			temp.setSourceId(entry.getSourceId());
			temp.setTaskId(entry.getTaskId());
			//url不为空才能添加
			if(!"".equals(url)&&url!=null){
				articleList.add(temp);
			}
		}
		return articleList;
	}

	@Override
	public Set<String> findHelpPaginationUrl(Page page, Entry entry) {
		/*List<String> pgList = page.getHtml().$("#pagebar_container").links().all();// 存放分页list
		List<String> pageUrls = page.getHtml().$("#sogou_next").links().all();//抓【下一页】链接
		pgList.addAll(pageUrls);
		Set<String> resultSet = new HashSet<>();
		resultSet.addAll(pgList);
		return resultSet;*/
		List<String> pgList = new ArrayList<String>();// 存放分页list
		boolean flag = false;
		List<String> pageUrls = page.getHtml().$("#pagebar_container").links().all();
		for (int i = 0; i < pageUrls.size(); i++) {
			String nextUrl = page.getHtml().$("#sogou_next","href").toString();
			String prevUrl = page.getHtml().$("#sogou_prev","href").toString();
			if (!"".equals(nextUrl)&&nextUrl!=null) {//抓【下一页】链接
				pgList.add(nextUrl);
			}
			if ((!"".equals(prevUrl)&&prevUrl!=null)||(!"".equals(nextUrl)&&nextUrl!=null)) {//包含上一页或者下一页链接
				flag = true;
			}
		}
		if (!flag) {//没有上一页链接也没有下一页链接 ，取全部
			pgList.addAll(pageUrls);
		}
		Set<String> resultSet = new HashSet<>();
		resultSet.addAll(pgList);
		return resultSet;
	}
}
